\documentclass{article}

\usepackage[paperwidth=16cm, paperheight=13.8cm,top=0.1cm,bottom=0.1cm,left=0.1cm,right=0.1cm]{geometry}
\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{amsmath}
\pagenumbering{gobble}
\begin{document}
\begin{algorithm}
	\caption{Vanilla Policy Gradient (Timestep Based)}
	\begin{algorithmic}[1]
		\STATE Randomly initialize policy network $\pi_\theta$: $\theta\gets \theta_0$, value function network $V_\phi$: $\phi\gets \phi_0$
		\REPEAT
		\STATE Collect $N$ trajectories $\mathcal{D}=\{\tau_0,\tau_1,\ldots,\tau_{N}\}$ by running policy $\pi_\theta$ in the environment, and store all transitions $\{ s_t,a_t,r_{t},s_{t+1},d_{t} \}$ in $N$ buffers. All trajectories have the same length $T$. If one episode ends early, reset environment and continue.
		\STATE Compute return at each time step along these trajectories:
		$$
			G_t=
			\left\{
			\begin{aligned}
				 & r_{t},\quad t=T                          \\
				 & r_{t}+\gamma G_{t+1},\quad t=T-1,\dots,1
			\end{aligned}
			\right.
		$$
		while $\tau\in\mathcal{D}$.
		\STATE Compute advantage at each time step $\hat{A}_t$ based on current value function $V_{\phi}$ using any method of advantage estimation (such as \textbf{TD Residual}):
		$$
			\hat{A}_t=r_{}+(1-d_{t+1})\cdot\gamma V_\phi(s_{t+1})-V_\phi(s_{t})
		$$
		\STATE Compute policy loss based on \textbf{Policy Gradient}, and update policy parameter $\theta$ via any gradient \textbf{ascent} algorithm:
		$$
			\begin{aligned}
				J(\theta) & =\frac{1}{N} \sum_{\tau\in \mathcal{D}}\sum^{T}_{t=0}\log\pi_\theta(a_t|s_t)\cdot \hat{A}_t \\
				\theta    & \gets \theta+\alpha \nabla_{\theta} J(\theta)
			\end{aligned}
		$$
		\STATE Compute value function loss, and update value function parameter $\phi$ via any gradient \textbf{descent} algorithm:
		$$
			\begin{aligned}
				L_V  & =\frac{1}{N}\sum_{\tau \in \mathcal{D}}\sum^{T}_{t=0}\left( G_t-V_\phi(s_t) \right)^2 \\
				\phi & \gets \phi+\beta\nabla_\phi L_V
			\end{aligned}
		$$
		\STATE Clear all buffers to empty.
		\UNTIL{convergence}

	\end{algorithmic}
\end{algorithm}
\end{document}
